{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature: Out-Of-Fold Predictions from a Bidirectional LSTM (+Magic Inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition to the bidirectional LSTM architecture, we'll append some of the leaky features to the intermediate feature layer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"assets/bi-lstm-with-magic.png\" alt=\"Network Architecture\" style=\"height: 700px;\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This utility package imports `numpy`, `pandas`, `matplotlib` and a helper `kg` module into the root namespace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pygoose import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import *\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Automatically discover the paths to various data folders and compose the project structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "project = kg.Project.discover()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Identifier for storing these features on disk and referring to them later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list_id = 'oofp_nn_bi_lstm_with_magic'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make subsequent NN runs reproducible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word embedding lookup matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_matrix = kg.io.load(project.aux_dir + 'fasttext_vocab_embedding_matrix.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Padded sequences of word indices for every question."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_q1 = kg.io.load(project.preprocessed_data_dir + 'sequences_q1_fasttext_train.pickle')\n",
    "X_train_q2 = kg.io.load(project.preprocessed_data_dir + 'sequences_q2_fasttext_train.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_q1 = kg.io.load(project.preprocessed_data_dir + 'sequences_q1_fasttext_test.pickle')\n",
    "X_test_q2 = kg.io.load(project.preprocessed_data_dir + 'sequences_q2_fasttext_test.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = kg.io.load(project.features_dir + 'y_train.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Magic features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "magic_feature_lists = [\n",
    "    'magic_frequencies',\n",
    "    'magic_cooccurrence_matrix',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_magic, X_test_magic, _ = project.load_feature_lists(magic_feature_lists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_magic = X_train_magic.values\n",
    "X_test_magic = X_test_magic.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler.fit(np.vstack([X_train_magic, X_test_magic]))\n",
    "X_train_magic = scaler.transform(X_train_magic)\n",
    "X_test_magic = scaler.transform(X_test_magic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word embedding properties."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = embedding_matrix.shape[-1]\n",
    "VOCAB_LENGTH = embedding_matrix.shape[0]\n",
    "MAX_SEQUENCE_LENGTH = X_train_q1.shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300 101564 30\n"
     ]
    }
   ],
   "source": [
    "print(EMBEDDING_DIM, VOCAB_LENGTH, MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model_question_branch():\n",
    "    input_q = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    \n",
    "    embedding_q = Embedding(\n",
    "        VOCAB_LENGTH,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        trainable=False,\n",
    "    )(input_q)\n",
    "\n",
    "    timedist_q = TimeDistributed(Dense(\n",
    "        EMBEDDING_DIM,\n",
    "        activation='relu',\n",
    "    ))(embedding_q)\n",
    "\n",
    "    lambda_q = Lambda(\n",
    "        lambda x: K.max(x, axis=1),\n",
    "        output_shape=(EMBEDDING_DIM, )\n",
    "    )(timedist_q)\n",
    "    \n",
    "    output_q = lambda_q\n",
    "    return input_q, output_q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model(params):    \n",
    "    embedding_layer = Embedding(\n",
    "        VOCAB_LENGTH,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        trainable=False,\n",
    "    )\n",
    "    lstm_layer = Bidirectional(LSTM(\n",
    "        params['num_lstm'],\n",
    "        dropout=params['lstm_dropout_rate'],\n",
    "        recurrent_dropout=params['lstm_dropout_rate'],\n",
    "    ))\n",
    "\n",
    "    sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences_1 = embedding_layer(sequence_1_input)\n",
    "    x1 = lstm_layer(embedded_sequences_1)\n",
    "\n",
    "    sequence_2_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences_2 = embedding_layer(sequence_2_input)\n",
    "    y1 = lstm_layer(embedded_sequences_2)\n",
    "\n",
    "    magic_input = Input(shape=(X_train_magic.shape[-1], ))\n",
    "    \n",
    "    merged = concatenate([x1, y1, magic_input])\n",
    "    merged = Dropout(params['dense_dropout_rate'])(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "\n",
    "    merged = Dense(params['num_dense'], activation='relu')(merged)\n",
    "    merged = Dropout(params['dense_dropout_rate'])(merged)\n",
    "    merged = BatchNormalization()(merged)\n",
    "\n",
    "    output = Dense(1, activation='sigmoid')(merged)\n",
    "\n",
    "    model = Model(\n",
    "        inputs=[sequence_1_input, sequence_2_input, magic_input],\n",
    "        outputs=output\n",
    "    )\n",
    "\n",
    "    model.compile(\n",
    "        loss='binary_crossentropy',\n",
    "        optimizer='nadam',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(model, X_q1, X_q2, X_magic):\n",
    "    \"\"\"\n",
    "    Mirror the pairs, compute two separate predictions, and average them.\n",
    "    \"\"\"\n",
    "    \n",
    "    y1 = model.predict([X_q1, X_q2, X_magic], batch_size=1024, verbose=1).reshape(-1)   \n",
    "    y2 = model.predict([X_q2, X_q1, X_magic], batch_size=1024, verbose=1).reshape(-1)    \n",
    "    return (y1 + y2) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Partition the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_FOLDS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(\n",
    "    n_splits=NUM_FOLDS,\n",
    "    shuffle=True,\n",
    "    random_state=RANDOM_SEED\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create placeholders for out-of-fold predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train_oofp = np.zeros_like(y_train, dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test_oofp = np.zeros((len(X_test_q1), NUM_FOLDS))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 2048"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_EPOCHS = 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best values picked by Bayesian optimization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_params = {\n",
    "    'dense_dropout_rate': 0.075,\n",
    "    'lstm_dropout_rate': 0.332,\n",
    "    'num_dense': 130,\n",
    "    'num_lstm': 300,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The path where the best weights of the current model will be saved."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_checkpoint_path = project.temp_dir + 'fold-checkpoint-' + feature_list_id + '.h5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the folds and compute out-of-fold predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fitting fold 1 of 5\n",
      "\n",
      "Train on 646862 samples, validate on 161718 samples\n",
      "Epoch 1/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.4096 - acc: 0.8138Epoch 00000: val_loss improved from inf to 0.37960, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 268s - loss: 0.4095 - acc: 0.8139 - val_loss: 0.3796 - val_acc: 0.8486\n",
      "Epoch 2/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.3405 - acc: 0.8453Epoch 00001: val_loss improved from 0.37960 to 0.31875, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 270s - loss: 0.3405 - acc: 0.8453 - val_loss: 0.3188 - val_acc: 0.8606\n",
      "Epoch 3/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.3136 - acc: 0.8575Epoch 00002: val_loss improved from 0.31875 to 0.29682, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 271s - loss: 0.3135 - acc: 0.8575 - val_loss: 0.2968 - val_acc: 0.8681\n",
      "Epoch 4/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2952 - acc: 0.8658Epoch 00003: val_loss improved from 0.29682 to 0.29668, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 262s - loss: 0.2952 - acc: 0.8659 - val_loss: 0.2967 - val_acc: 0.8639\n",
      "Epoch 5/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2802 - acc: 0.8736Epoch 00004: val_loss improved from 0.29668 to 0.28034, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 265s - loss: 0.2802 - acc: 0.8736 - val_loss: 0.2803 - val_acc: 0.8740\n",
      "Epoch 6/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2680 - acc: 0.8788Epoch 00005: val_loss improved from 0.28034 to 0.27428, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 272s - loss: 0.2680 - acc: 0.8788 - val_loss: 0.2743 - val_acc: 0.8778\n",
      "Epoch 7/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2570 - acc: 0.8843Epoch 00006: val_loss improved from 0.27428 to 0.27257, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 272s - loss: 0.2570 - acc: 0.8844 - val_loss: 0.2726 - val_acc: 0.8786\n",
      "Epoch 8/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2474 - acc: 0.8889Epoch 00007: val_loss did not improve\n",
      "646862/646862 [==============================] - 262s - loss: 0.2474 - acc: 0.8889 - val_loss: 0.2817 - val_acc: 0.8722\n",
      "Epoch 9/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2395 - acc: 0.8928Epoch 00008: val_loss did not improve\n",
      "646862/646862 [==============================] - 269s - loss: 0.2394 - acc: 0.8929 - val_loss: 0.2796 - val_acc: 0.8750\n",
      "Epoch 10/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2309 - acc: 0.8972Epoch 00009: val_loss did not improve\n",
      "646862/646862 [==============================] - 271s - loss: 0.2309 - acc: 0.8972 - val_loss: 0.2800 - val_acc: 0.8743\n",
      "Epoch 11/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2226 - acc: 0.9013Epoch 00010: val_loss did not improve\n",
      "646862/646862 [==============================] - 270s - loss: 0.2226 - acc: 0.9013 - val_loss: 0.2831 - val_acc: 0.8737\n",
      "Epoch 00010: early stopping\n",
      "80859/80859 [==============================] - 9s     \n",
      "80859/80859 [==============================] - 8s     \n",
      "2345796/2345796 [==============================] - 240s   \n",
      "2345796/2345796 [==============================] - 241s   \n",
      "\n",
      "Fitting fold 2 of 5\n",
      "\n",
      "Train on 646862 samples, validate on 161718 samples\n",
      "Epoch 1/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.4103 - acc: 0.8130Epoch 00000: val_loss improved from inf to 0.40834, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 264s - loss: 0.4102 - acc: 0.8131 - val_loss: 0.4083 - val_acc: 0.8501\n",
      "Epoch 2/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.3388 - acc: 0.8464Epoch 00001: val_loss improved from 0.40834 to 0.32283, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 259s - loss: 0.3387 - acc: 0.8464 - val_loss: 0.3228 - val_acc: 0.8599\n",
      "Epoch 3/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.3118 - acc: 0.8588Epoch 00002: val_loss improved from 0.32283 to 0.29328, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 261s - loss: 0.3118 - acc: 0.8588 - val_loss: 0.2933 - val_acc: 0.8701\n",
      "Epoch 4/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2932 - acc: 0.8670Epoch 00003: val_loss improved from 0.29328 to 0.28802, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 262s - loss: 0.2932 - acc: 0.8670 - val_loss: 0.2880 - val_acc: 0.8701\n",
      "Epoch 5/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2790 - acc: 0.8740Epoch 00004: val_loss did not improve\n",
      "646862/646862 [==============================] - 263s - loss: 0.2790 - acc: 0.8740 - val_loss: 0.3003 - val_acc: 0.8626\n",
      "Epoch 6/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2672 - acc: 0.8795Epoch 00005: val_loss improved from 0.28802 to 0.28025, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646862/646862 [==============================] - 253s - loss: 0.2672 - acc: 0.8796 - val_loss: 0.2803 - val_acc: 0.8746\n",
      "Epoch 7/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2563 - acc: 0.8847Epoch 00006: val_loss did not improve\n",
      "646862/646862 [==============================] - 260s - loss: 0.2563 - acc: 0.8847 - val_loss: 0.2964 - val_acc: 0.8639\n",
      "Epoch 8/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2469 - acc: 0.8893Epoch 00007: val_loss did not improve\n",
      "646862/646862 [==============================] - 255s - loss: 0.2469 - acc: 0.8893 - val_loss: 0.2830 - val_acc: 0.8722\n",
      "Epoch 9/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2384 - acc: 0.8935Epoch 00008: val_loss did not improve\n",
      "646862/646862 [==============================] - 257s - loss: 0.2383 - acc: 0.8935 - val_loss: 0.2859 - val_acc: 0.8745\n",
      "Epoch 10/200\n",
      "645120/646862 [============================>.] - ETA: 0s - loss: 0.2306 - acc: 0.8972Epoch 00009: val_loss did not improve\n",
      "646862/646862 [==============================] - 259s - loss: 0.2307 - acc: 0.8972 - val_loss: 0.2910 - val_acc: 0.8718\n",
      "Epoch 00009: early stopping\n",
      "80859/80859 [==============================] - 9s     \n",
      "80859/80859 [==============================] - 8s     \n",
      "2345796/2345796 [==============================] - 240s   \n",
      "2345796/2345796 [==============================] - ETA:  - 240s   \n",
      "\n",
      "Fitting fold 3 of 5\n",
      "\n",
      "Train on 646864 samples, validate on 161716 samples\n",
      "Epoch 1/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.4114 - acc: 0.8125Epoch 00000: val_loss improved from inf to 0.37732, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 265s - loss: 0.4112 - acc: 0.8126 - val_loss: 0.3773 - val_acc: 0.8458\n",
      "Epoch 2/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.3414 - acc: 0.8453Epoch 00001: val_loss improved from 0.37732 to 0.33107, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 260s - loss: 0.3414 - acc: 0.8453 - val_loss: 0.3311 - val_acc: 0.8583\n",
      "Epoch 3/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.3130 - acc: 0.8583Epoch 00002: val_loss improved from 0.33107 to 0.30420, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 266s - loss: 0.3130 - acc: 0.8583 - val_loss: 0.3042 - val_acc: 0.8633\n",
      "Epoch 4/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2943 - acc: 0.8667Epoch 00003: val_loss did not improve\n",
      "646864/646864 [==============================] - 254s - loss: 0.2943 - acc: 0.8667 - val_loss: 0.3059 - val_acc: 0.8588\n",
      "Epoch 5/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2799 - acc: 0.8738Epoch 00004: val_loss improved from 0.30420 to 0.28269, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 260s - loss: 0.2799 - acc: 0.8738 - val_loss: 0.2827 - val_acc: 0.8721\n",
      "Epoch 6/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2684 - acc: 0.8789Epoch 00005: val_loss improved from 0.28269 to 0.27958, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 267s - loss: 0.2683 - acc: 0.8789 - val_loss: 0.2796 - val_acc: 0.8727\n",
      "Epoch 7/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2577 - acc: 0.8844Epoch 00006: val_loss improved from 0.27958 to 0.27245, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646864/646864 [==============================] - 274s - loss: 0.2577 - acc: 0.8844 - val_loss: 0.2724 - val_acc: 0.8775\n",
      "Epoch 8/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2478 - acc: 0.8895Epoch 00007: val_loss did not improve\n",
      "646864/646864 [==============================] - 271s - loss: 0.2478 - acc: 0.8895 - val_loss: 0.2925 - val_acc: 0.8657\n",
      "Epoch 9/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2391 - acc: 0.8933Epoch 00008: val_loss did not improve\n",
      "646864/646864 [==============================] - 273s - loss: 0.2392 - acc: 0.8932 - val_loss: 0.2813 - val_acc: 0.8722\n",
      "Epoch 10/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2307 - acc: 0.8974Epoch 00009: val_loss did not improve\n",
      "646864/646864 [==============================] - 271s - loss: 0.2307 - acc: 0.8974 - val_loss: 0.2822 - val_acc: 0.8744\n",
      "Epoch 11/200\n",
      "645120/646864 [============================>.] - ETA: 0s - loss: 0.2239 - acc: 0.9009Epoch 00010: val_loss did not improve\n",
      "646864/646864 [==============================] - 273s - loss: 0.2239 - acc: 0.9009 - val_loss: 0.2938 - val_acc: 0.8704\n",
      "Epoch 00010: early stopping\n",
      "80858/80858 [==============================] - 9s     \n",
      "80858/80858 [==============================] - 9s     \n",
      "2345796/2345796 [==============================] - 266s   \n",
      "2345796/2345796 [==============================] - 267s   \n",
      "\n",
      "Fitting fold 4 of 5\n",
      "\n",
      "Train on 646866 samples, validate on 161714 samples\n",
      "Epoch 1/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.4107 - acc: 0.8132Epoch 00000: val_loss improved from inf to 0.38107, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 272s - loss: 0.4105 - acc: 0.8133 - val_loss: 0.3811 - val_acc: 0.8540\n",
      "Epoch 2/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.3394 - acc: 0.8458Epoch 00001: val_loss improved from 0.38107 to 0.31872, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 266s - loss: 0.3393 - acc: 0.8459 - val_loss: 0.3187 - val_acc: 0.8616\n",
      "Epoch 3/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.3121 - acc: 0.8588Epoch 00002: val_loss improved from 0.31872 to 0.29530, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 273s - loss: 0.3121 - acc: 0.8588 - val_loss: 0.2953 - val_acc: 0.8666\n",
      "Epoch 4/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2944 - acc: 0.8668Epoch 00003: val_loss improved from 0.29530 to 0.28555, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 269s - loss: 0.2944 - acc: 0.8669 - val_loss: 0.2856 - val_acc: 0.8716\n",
      "Epoch 5/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2794 - acc: 0.8739Epoch 00004: val_loss did not improve\n",
      "646866/646866 [==============================] - 267s - loss: 0.2794 - acc: 0.8739 - val_loss: 0.2865 - val_acc: 0.8695\n",
      "Epoch 6/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2678 - acc: 0.8793Epoch 00005: val_loss improved from 0.28555 to 0.27500, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 264s - loss: 0.2678 - acc: 0.8793 - val_loss: 0.2750 - val_acc: 0.8771\n",
      "Epoch 7/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2574 - acc: 0.8843Epoch 00006: val_loss improved from 0.27500 to 0.27241, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 264s - loss: 0.2574 - acc: 0.8843 - val_loss: 0.2724 - val_acc: 0.8787\n",
      "Epoch 8/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2475 - acc: 0.8889Epoch 00007: val_loss did not improve\n",
      "646866/646866 [==============================] - 260s - loss: 0.2476 - acc: 0.8888 - val_loss: 0.2763 - val_acc: 0.8759\n",
      "Epoch 9/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2387 - acc: 0.8933Epoch 00008: val_loss did not improve\n",
      "646866/646866 [==============================] - 258s - loss: 0.2387 - acc: 0.8933 - val_loss: 0.2749 - val_acc: 0.8769\n",
      "Epoch 10/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2316 - acc: 0.8966Epoch 00009: val_loss did not improve\n",
      "646866/646866 [==============================] - 260s - loss: 0.2316 - acc: 0.8966 - val_loss: 0.2779 - val_acc: 0.8775\n",
      "Epoch 11/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2240 - acc: 0.9001Epoch 00010: val_loss did not improve\n",
      "646866/646866 [==============================] - 261s - loss: 0.2241 - acc: 0.9001 - val_loss: 0.2944 - val_acc: 0.8680\n",
      "Epoch 00010: early stopping\n",
      "80857/80857 [==============================] - 9s     \n",
      "80857/80857 [==============================] - 8s     \n",
      "2345796/2345796 [==============================] - 259s   \n",
      "2345796/2345796 [==============================] - 260s   \n",
      "\n",
      "Fitting fold 5 of 5\n",
      "\n",
      "Train on 646866 samples, validate on 161714 samples\n",
      "Epoch 1/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.4099 - acc: 0.8129Epoch 00000: val_loss improved from inf to 0.40600, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 262s - loss: 0.4098 - acc: 0.8129 - val_loss: 0.4060 - val_acc: 0.8487\n",
      "Epoch 2/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.3414 - acc: 0.8454Epoch 00001: val_loss improved from 0.40600 to 0.33487, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "646866/646866 [==============================] - 261s - loss: 0.3414 - acc: 0.8454 - val_loss: 0.3349 - val_acc: 0.8516\n",
      "Epoch 3/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.3137 - acc: 0.8581Epoch 00002: val_loss improved from 0.33487 to 0.29464, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 261s - loss: 0.3137 - acc: 0.8581 - val_loss: 0.2946 - val_acc: 0.8681\n",
      "Epoch 4/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2951 - acc: 0.8661Epoch 00003: val_loss improved from 0.29464 to 0.28505, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 265s - loss: 0.2952 - acc: 0.8661 - val_loss: 0.2850 - val_acc: 0.8728\n",
      "Epoch 5/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2806 - acc: 0.8731Epoch 00004: val_loss improved from 0.28505 to 0.28255, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 262s - loss: 0.2806 - acc: 0.8731 - val_loss: 0.2826 - val_acc: 0.8732\n",
      "Epoch 6/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2687 - acc: 0.8784Epoch 00005: val_loss improved from 0.28255 to 0.27996, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 262s - loss: 0.2687 - acc: 0.8784 - val_loss: 0.2800 - val_acc: 0.8731\n",
      "Epoch 7/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2579 - acc: 0.8842Epoch 00006: val_loss did not improve\n",
      "646866/646866 [==============================] - 262s - loss: 0.2578 - acc: 0.8842 - val_loss: 0.2880 - val_acc: 0.8682\n",
      "Epoch 8/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2491 - acc: 0.8882Epoch 00007: val_loss improved from 0.27996 to 0.27319, saving model to /home/yuriyguts/Projects/kaggle-quora-question-pairs/data/tmp/fold-checkpoint-oofp_nn_bi_lstm_with_magic.h5\n",
      "646866/646866 [==============================] - 262s - loss: 0.2490 - acc: 0.8882 - val_loss: 0.2732 - val_acc: 0.8773\n",
      "Epoch 9/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2393 - acc: 0.8931Epoch 00008: val_loss did not improve\n",
      "646866/646866 [==============================] - 266s - loss: 0.2393 - acc: 0.8931 - val_loss: 0.2740 - val_acc: 0.8776\n",
      "Epoch 10/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2313 - acc: 0.8973Epoch 00009: val_loss did not improve\n",
      "646866/646866 [==============================] - 261s - loss: 0.2312 - acc: 0.8973 - val_loss: 0.2842 - val_acc: 0.8728\n",
      "Epoch 11/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2243 - acc: 0.9006Epoch 00010: val_loss did not improve\n",
      "646866/646866 [==============================] - 262s - loss: 0.2243 - acc: 0.9006 - val_loss: 0.2952 - val_acc: 0.8692\n",
      "Epoch 12/200\n",
      "645120/646866 [============================>.] - ETA: 0s - loss: 0.2179 - acc: 0.9034Epoch 00011: val_loss did not improve\n",
      "646866/646866 [==============================] - 259s - loss: 0.2179 - acc: 0.9034 - val_loss: 0.2815 - val_acc: 0.8752\n",
      "Epoch 00011: early stopping\n",
      "80857/80857 [==============================] - 9s     \n",
      "80857/80857 [==============================] - 9s     \n",
      "2345796/2345796 [==============================] - 257s   \n",
      "2345796/2345796 [==============================] - 257s   \n",
      "CPU times: user 3h 21min 32s, sys: 32min 46s, total: 3h 54min 18s\n",
      "Wall time: 4h 47min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Iterate through folds.\n",
    "for fold_num, (ix_train, ix_val) in enumerate(kfold.split(X_train_q1, y_train)):\n",
    "    \n",
    "    # Augment the training set by mirroring the pairs.\n",
    "    X_fold_train_q1 = np.vstack([X_train_q1[ix_train], X_train_q2[ix_train]])\n",
    "    X_fold_train_q2 = np.vstack([X_train_q2[ix_train], X_train_q1[ix_train]])\n",
    "    X_fold_train_magic = np.vstack([X_train_magic[ix_train], X_train_magic[ix_train]])\n",
    "\n",
    "    X_fold_val_q1 = np.vstack([X_train_q1[ix_val], X_train_q2[ix_val]])\n",
    "    X_fold_val_q2 = np.vstack([X_train_q2[ix_val], X_train_q1[ix_val]])\n",
    "    X_fold_val_magic = np.vstack([X_train_magic[ix_val], X_train_magic[ix_val]])\n",
    "\n",
    "    # Ground truth should also be \"mirrored\".\n",
    "    y_fold_train = np.concatenate([y_train[ix_train], y_train[ix_train]])\n",
    "    y_fold_val = np.concatenate([y_train[ix_val], y_train[ix_val]])\n",
    "    \n",
    "    print()\n",
    "    print(f'Fitting fold {fold_num + 1} of {kfold.n_splits}')\n",
    "    print()\n",
    "    \n",
    "    # Compile a new model.\n",
    "    model = create_model(model_params)\n",
    "\n",
    "    # Train.\n",
    "    model.fit(\n",
    "        [X_fold_train_q1, X_fold_train_q2, X_fold_train_magic], y_fold_train,\n",
    "        validation_data=([X_fold_val_q1, X_fold_val_q2, X_fold_val_magic], y_fold_val),\n",
    "\n",
    "        batch_size=BATCH_SIZE,\n",
    "        epochs=MAX_EPOCHS,\n",
    "        verbose=1,\n",
    "        \n",
    "        callbacks=[\n",
    "            # Stop training when the validation loss stops improving.\n",
    "            EarlyStopping(\n",
    "                monitor='val_loss',\n",
    "                min_delta=0.001,\n",
    "                patience=3,\n",
    "                verbose=1,\n",
    "                mode='auto',\n",
    "            ),\n",
    "            # Save the weights of the best epoch.\n",
    "            ModelCheckpoint(\n",
    "                model_checkpoint_path,\n",
    "                monitor='val_loss',\n",
    "                save_best_only=True,\n",
    "                verbose=2,\n",
    "            ),\n",
    "        ],\n",
    "    )\n",
    "        \n",
    "    # Restore the best epoch.\n",
    "    model.load_weights(model_checkpoint_path)\n",
    "    \n",
    "    # Compute out-of-fold predictions.\n",
    "    y_train_oofp[ix_val] = predict(model, X_train_q1[ix_val], X_train_q2[ix_val], X_train_magic[ix_val])\n",
    "    y_test_oofp[:, fold_num] = predict(model, X_test_q1, X_test_q2, X_test_magic)\n",
    "    \n",
    "    # Clear GPU memory.\n",
    "    K.clear_session()\n",
    "    del X_fold_train_q1, X_fold_train_q2, X_fold_train_magic\n",
    "    del X_fold_val_q1, X_fold_val_q2, X_fold_val_magic\n",
    "    del model\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV score: 0.271838110004\n"
     ]
    }
   ],
   "source": [
    "cv_score = log_loss(y_train, y_train_oofp)\n",
    "print('CV score:', cv_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_names = [feature_list_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_train = y_train_oofp.reshape((-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features_test = np.mean(y_test_oofp, axis=1).reshape((-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "project.save_features(features_train, features_test, feature_names, feature_list_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fe182c28400>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqAAAAGoCAYAAACZh1c1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90lPWZ///XTBIyM9BCQkBLy8YwtAtWSCSARCkg2dbT\nNiEK8ahRpLhoglDWVqimLA0WKHYl2QXbRfSApwIVJa4EUnQpKogLqUREFITaZordZakZI1Emkx9m\n7u8ffpmPs4EwkuQ9P/J8nNPTw/2+M9c1Xia8fN+577FZlmUJAAAAMMQe6QYAAADQuxBAAQAAYBQB\nFAAAAEYRQAEAAGAUARQAAABGEUABAABgVGKkG4hH9fWf9HgNm82mgQP76sMPfeJJWtGLOUU/ZhQb\nmFP0Y0axweScBg360gXX2AGNUXb7Z/8S2ZlgVGNO0Y8ZxQbmFP2YUWyIljnxrwkAAACMIoACAADA\nKAIoAAAAjCKAAgAAwCgCKAAAAIwigAIAAMAoAigAAACMIoACAADAKAIoAAAAjCKAAgAAwCg+Cx4A\nACBC7nr4ZSN1Njw49Qud/8c/Htcjj/xCHk+dvva1v9PChaW66qpR3dYPO6AAAAAIamlp0QMP/Fjf\n+940vfjiHhUW3qIHH/yxmpqauq0GARQAAABBhw7Vymaz6aabCpWYmKi8vAKlpqbqwIH/6rYaBFAA\nAAAEvf/+X3TFFcNCjv3d36Xr/ff/0m01CKAAAAAI8vv9cjgcIceSkx1qbm7uthoEUAAAAAQ5HA61\ntLSEHGtpaZbT6ey2GgRQAAAABKWnZ+j990+GHHv//ZPKyBh2ga/44ngMUwzLv78q0i10uy/6mAgA\nANC9srPHqa2tVZWVW3TjjYV68cXfqaGhQePH53RbDXZAAQAAENSnTx+tWrVGu3fv0ne/O1XPPfeM\nHn64olsvwbMDCgAAECHReuVv+PCv67HHNvTY67MDCgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAK\nAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMCoiAfTIkSOa\nOHFi8M+nT5/Wvffeq2uuuUbXXXedli1bptbWVkmSZVkqLy/XhAkTNG7cOC1fvlzt7e3Br62urlZu\nbq6ysrJUXFwsr9cbXDt27JgKCwuVlZWlgoICHT58OLjW2NioefPmKTs7W1OmTNHWrVuDaxerCQAA\ngEtnNIBalqXKykrdddddamtrCx5ftGiRLr/8cr366qvatm2b3n77bf3617+WJG3evFl79uzR9u3b\ntXPnTh06dEgbNmyQJB0/flxlZWWqqKhQTU2N0tLSVFpaKklqaWlRSUmJpk+froMHD2rmzJmaO3eu\nfD6fJGnJkiVyuVzav3+/1qxZo1WrVgUDamc1AQAA0DVGA+hjjz2mp556SiUlJcFjra2tcjqdmjt3\nrpKTkzVo0CDl5+frzTfflCRVVVVp1qxZGjx4sAYNGqTi4mI9//zzkqQdO3YoNzdXmZmZcjgcWrhw\nofbt2yev16uamhrZ7XYVFRUpKSlJhYWFSktL0969e+Xz+bR7924tWLBAycnJGj16tPLy8rRt27aL\n1gQAAEDXGA2gM2bMUFVVlUaNGhU81qdPHz3++OMaNGhQ8Ngrr7yiESNGSJLq6uo0fPjw4FpGRoY8\nHo8sy+qwlpKSov79+8vj8cjj8cjtdofUz8jIUF1dnU6ePKnExEQNHTq0w9rFagIAAKBrEk0WGzx4\ncKfrlmVpxYoVqqur0yOPPCJJ8vv9cjgcwXOcTqcCgYBaW1s7rJ1b9/v9ampqktPpDFlzOBxqbm5W\nU1NTh687t3axmsnJyRd9nzabTfYejvZ2u61nC0RIQkJ8va9zc4rXecUDZhQbmFP0Y0axIVrmZDSA\ndqa5uVk/+clPdOLECW3cuFEDBw6U9FkwbGlpCZ7n9/uVmJio5OTkkND4+XWXyyWn09lhrbm5Obj2\n+df8/NrFaoZj4MC+stn4BrwUqan9It1CjxgwoG+kW8BFMKPYwJyiHzOKDZGeU1QE0DNnzmjOnDly\nuVx65plnNGDAgOCa2+2Wx+NRZmamJMnj8WjYsGEha+c0NDSosbFRbrdbPp9PmzZtCqnj8XiUl5en\n9PR0tbW16dSpUxoyZEhw7dxl985qhuPDD33sgF6ihoazkW6hW9ntNg0Y0FdnzvgUCPArHNGIGcUG\n5hT9mFFsMDmnzjaVIh5ALcvSD3/4Q6WlpenRRx9VUlJSyPq0adO0fv16TZgwQYmJiVq3bp0KCgok\nSXl5ebrjjjs0Y8YMjRo1ShUVFZo0aZJSUlKUk5Oj1tZWbdy4Ubfeequqqqrk9Xo1ceJEuVwu5ebm\nqry8XMuXL9d7772n6upqPf744xetGe574qlNl6a9PT5/aAUCVty+t3jBjGIDc4p+zCg2RHpOEQ+g\nb775pl5//XUlJydr/PjxweNXXnmlNm/erKKiInm9XhUWFqqtrU35+fmaPXu2JGnkyJFatmyZFi9e\nrPr6eo0dO1YrV66U9NnNTU888YSWLl2qiooKpaena+3atcHL7MuWLVNZWZkmT54sl8ulRYsWBXc8\nO6sJAACArrFZ3Nrd7errP+nxGgkJNs1a8VKP1zFtw4NTI91Ct0pIsCk1tZ8aGs6yIxClmFFsYE7R\njxnFBpNzGjToSxdc46M4AQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAA\nYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQA\nAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFA\nAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAU\nARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAA\nRhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYFREAuiRI0c0ceLE\n4J8bGxs1b948ZWdna8qUKdq6dWtwzbIslZeXa8KECRo3bpyWL1+u9vb24Hp1dbVyc3OVlZWl4uJi\neb3e4NqxY8dUWFiorKwsFRQU6PDhw91SEwAAAJfOaAC1LEuVlZW666671NbWFjy+ZMkSuVwu7d+/\nX2vWrNGqVauCYXHz5s3as2ePtm/frp07d+rQoUPasGGDJOn48eMqKytTRUWFampqlJaWptLSUklS\nS0uLSkpKNH36dB08eFAzZ87U3Llz5fP5ulQTAAAAXWM0gD722GN66qmnVFJSEjzm8/m0e/duLViw\nQMnJyRo9erTy8vK0bds2SVJVVZVmzZqlwYMHa9CgQSouLtbzzz8vSdqxY4dyc3OVmZkph8OhhQsX\nat++ffJ6vaqpqZHdbldRUZGSkpJUWFiotLQ07d27t0s1AQAA0DWJJovNmDFDJSUlev3114PHTp48\nqcTERA0dOjR4LCMjQ7t27ZIk1dXVafjw4SFrHo9HlmWprq5OV199dXAtJSVF/fv3l8fjkcfjkdvt\nDqmfkZGhuro6XXHFFZdc02azXfR92mw22Xs42tvtF+8jFiUkxNf7OjeneJ1XPGBGsYE5RT9mFBui\nZU5GA+jgwYM7HGtqapLD4Qg55nA41NzcLEny+/0h606nU4FAQK2trR3Wzq37/X41NTXJ6XSe93W7\nUjM5Ofmi73PgwL5hBVV0lJraL9It9IgBA/pGugVcBDOKDcwp+jGj2BDpORkNoOfjdDrV0tIScqy5\nuVkul0vSZ8Hw8+t+v1+JiYlKTk4OCY2fX3e5XHI6nR3Wzr1uV2qG48MPfeyAXqKGhrORbqFb2e02\nDRjQV2fO+BQIWJFuB+fBjGIDc4p+zCg2mJxTZ5tKEQ+g6enpamtr06lTpzRkyBBJksfjCV4Cd7vd\n8ng8yszMDK4NGzYsZO2choYGNTY2yu12y+fzadOmTSG1PB6P8vLyulQzHJZliZvmL017e3z+0AoE\nrLh9b/GCGcUG5hT9mFFsiPScIv4c0H79+ik3N1fl5eXy+/06cuSIqqurlZ+fL0maNm2a1q9fr9On\nT8vr9WrdunUqKCiQJOXl5WnXrl2qra1VS0uLKioqNGnSJKWkpCgnJ0etra3auHGj2traVFlZKa/X\nq4kTJ3apJgAAALom4jugkrRs2TKVlZVp8uTJcrlcWrRoUXD3saioSF6vV4WFhWpra1N+fr5mz54t\nSRo5cqSWLVumxYsXq76+XmPHjtXKlSslSX369NETTzyhpUuXqqKiQunp6Vq7dm3wMvul1gQAAEDX\n2CzLYp+8m9XXf9LjNRISbJq14qUer2PahgenRrqFbpWQYFNqaj81NJzlklSUYkaxgTlFP2YUG0zO\nadCgL11wLeKX4AEAANC7EEABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUAB\nAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQB\nFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABG\nEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAA\nYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQA\nAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGBU1ATQQ4cOafr06RozZoxuuOEG7dix\nQ5LU2NioefPmKTs7W1OmTNHWrVuDX2NZlsrLyzVhwgSNGzdOy5cvV3t7e3C9urpaubm5ysrKUnFx\nsbxeb3Dt2LFjKiwsVFZWlgoKCnT48OHgWmc1AQAA0DVREUDb29s1b9483XPPPTp06JBWrFihBx98\nUP/93/+tJUuWyOVyaf/+/VqzZo1WrVoVDIubN2/Wnj17tH37du3cuVOHDh3Shg0bJEnHjx9XWVmZ\nKioqVFNTo7S0NJWWlkqSWlpaVFJSounTp+vgwYOaOXOm5s6dK5/PJ0md1gQAAEDXREUA/fjjj9XQ\n0KD29nZZliWbzaakpCQlJCRo9+7dWrBggZKTkzV69Gjl5eVp27ZtkqSqqirNmjVLgwcP1qBBg1Rc\nXKznn39ekrRjxw7l5uYqMzNTDodDCxcu1L59++T1elVTUyO73a6ioiIlJSWpsLBQaWlp2rt3r3w+\nX6c1AQAA0DWJkW5AklJSUlRUVKQf//jHWrRokQKBgFasWKGPPvpIiYmJGjp0aPDcjIwM7dq1S5JU\nV1en4cOHh6x5PB5ZlqW6ujpdffXVITX69+8vj8cjj8cjt9sd0kNGRobq6up0xRVXdFozHDabTfYe\njvZ2u61nC0RIQkJ8va9zc4rXecUDZhQbmFP0Y0axIVrmFBUBNBAIyOFwaPXq1Zo6dar279+v+++/\nX2vXrpXD4Qg51+FwqLm5WZLk9/tD1p1OpwKBgFpbWzusnVv3+/1qamqS0+k87+s2NTV1WjMcAwf2\nlc3GN+ClSE3tF+kWesSAAX0j3QIughnFBuYU/ZhRbIj0nKIigO7atUtHjhzRAw88IEmaMmWKpkyZ\nokcffVQtLS0h5zY3N8vlckn6LBh+ft3v9ysxMVHJycnnDY1+v18ul0tOp7PD2rnXdTqdndYMx4cf\n+tgBvUQNDWcj3UK3stttGjCgr86c8SkQsCLdDs6DGcUG5hT9mFFsMDmnzjaVoiKA/u///q9aW1tD\njiUmJuqb3/ym3njjDZ06dUpDhgyRJHk8nuBld7fbLY/Ho8zMzODasGHDQtbOaWhoUGNjo9xut3w+\nnzZt2hRSz+PxKC8vT+np6Wpra7tgzXBYlqXP3YyPL6C9PT5/aAUCVty+t3jBjGIDc4p+zCg2RHpO\nUXET0rXXXqt3331Xzz33nCzL0uuvv67f//73+v73v6/c3FyVl5fL7/fryJEjqq6uVn5+viRp2rRp\nWr9+vU6fPi2v16t169apoKBAkpSXl6ddu3aptrZWLS0tqqio0KRJk5SSkqKcnBy1trZq48aNamtr\nU2VlpbxeryZOnKh+/fp1WhMAAABdY7MsKyr+M+Xll1/W6tWr9de//lVDhgzRP/3TP+nb3/62zpw5\no7KyMh04cEAul0vz589XYWGhpM8e37RmzRo999xzamtrU35+vkpLS5WQkCBJ2rlzp1avXq36+nqN\nHTtWK1eu1MCBAyV99pimpUuX6sSJE0pPT9fSpUuVlZUlSZ3WDEd9/Sfd/E+no4QEm2ateKnH65i2\n4cGpkW6hWyUk2JSa2k8NDWfZEYhSzCg2MKfox4xig8k5DRr0pQuuRU0AjScE0EtHAIVpzCg2MKfo\nx4xiQ7QE0Ki4BA8AAIDegwAKAAAAowigAAAAMCrsAFpQUBC84xwAAAC4VGEH0Jtvvlm7d+9Wbm6u\nZs6cqWeeeUaNjY092RsAAADiUNgB9I477tDTTz+tXbt2adKkSXrmmWf0rW99S/fee69eeOGFDg+S\nBwAAAM7nC38S0le/+lXdfffdmjZtmrZs2aINGzbo5ZdfVr9+/XTjjTfqhz/8ofr3798TvQIAACAO\nfKGbkLxerzZt2qSioiJdf/312rt3r+677z69+uqr+s1vfqNjx46ppKSkp3oFAABAHAh7B/TOO+/U\nG2+8ocsvv1x5eXlatmyZ3G53cH3w4MG68847tXjx4h5pFAAAAPEh7ADqdrt13333acyYMRc8Z/z4\n8dq2bVu3NAYAAID4FPYl+LKyMtXX1+uVV14JHluyZIl2794d/HNqaqqGDh3avR0CAAAgroQdQJ98\n8kmVlpbqzJkzwWNf/vKX9cADD2jLli090hwAAADiT9gBdOPGjSovL9dNN90UPLZo0SL98pe/1Pr1\n63ukOQAAAMSfsAPoRx99pPT09A7Hhw8frg8++KBbmwIAAED8CjuAZmZmav369Wpvbw8esyxLTz31\nlK688soeaQ4AAADxJ+y74B988EH94Ac/0GuvvaaRI0dKkk6cOKHW1lY9/vjjPdYgAAAA4kvYAXTE\niBF64YUXtHPnTv35z39WUlKSJk+erPz8fPXr168newQAAEAc+UIfxZmSkqLbb7+9p3oBAABALxB2\nAH3//fe1atUqvfPOO2pra5NlWSHrr732Wrc3BwAAgPgTdgAtLS1VQ0ODZs+ezSV3AAAAXLKwA+jb\nb7+tyspKfeMb3+jJfgAAABDnwn4M05AhQ3T27Nme7AUAAAC9QNg7oPfff78eeughzZ8/X+np6UpK\nSgpZz8jI6PbmAAAAEH/CDqA//OEPQ/5fkmw2myzLks1m07vvvtv93QEAACDuhB1AX3rppZ7sAwAA\nAL1E2AH0q1/9qiTpb3/7mzwej7KysnT27FmlpaX1WHMAAACIP2HfhNTU1KT77rtPkydP1l133aX6\n+nr97Gc/U1FRkRoaGnqyRwAAAMSRsAPoI488or/97W964YUXlJycLOmzG5NaWlr0i1/8oscaBAAA\nQHwJO4C+9NJLKi0tDbnb3e1266GHHtK+fft6pDkAAADEn7AD6NmzZ8/7CUh2u12ffvpptzYFAACA\n+BV2AJ04caIee+wxtbe3B4999NFHeuSRR3Tdddf1SHMAAACIP2EH0H/+53/WX/7yF+Xk5Ki5uVlz\n5szR9ddfr8bGRi1evLgnewQAAEAcCfsxTIMHD9azzz6rAwcOqK6uTp9++qncbreuu+462Wy2nuwR\nAAAAcSTsAHpOTk6OcnJyeqIXAAAA9AJhB9ARI0Z0utPJR3ECAAAgHGEH0CeeeCLkz+3t7Xr//fe1\nceNG/ehHP+r2xgAAABCfwg6g3/rWt857fPjw4SovL9f3vve9bmsKAAAA8Svsu+Av5Ctf+Yree++9\n7ugFAAAAvUDYO6CvvfZah2Nnz57V5s2bNWLEiG5tCgAAAPEr7AA6Z86cDseSkpI0atQo/fznP+/W\npgAAABC/wg6gx48f78k+AAAA0EuEHUA9Hk/YL5qRkXFJzQAAACD+hR1Av/vd7wafA2pZliR1eC6o\nZVmy2Ww8ExQAAAAXFHYAffTRR1VRUaFFixYpOztbSUlJOnr0qJYtW6bp06fr29/+dk/2CQAAgDgR\ndgBduXKl/uVf/kVjx44NHhs3bpyWL1+u+fPn6wc/+EFP9AcAAIA4E/ZzQD/++GP16dOnw/HW1lb5\n/f5ubQoAAADxK+wA+u1vf1s//elPtX//fn300UdqaGjQnj17tHjxYt1444092SMAAADiSNiX4Jcs\nWaLFixfr7rvvViAQkPTZc0Bnzpyp++67r8caBAAAQHwJewfU5XLpX//1X3XgwAE988wz2r59uw4e\nPKhFixYpKSmpy42cPn1axcXFGjNmjCZNmqSnnnpKktTY2Kh58+YpOztbU6ZM0datW4NfY1mWysvL\nNWHChODvo7a3twfXq6urlZubq6ysLBUXF8vr9QbXjh07psLCQmVlZamgoECHDx8OrnVWEwAAAF3z\nhT4L/sMPP9Rvf/tb/fa3v1Vqaqpeeukl/fGPf+xyE5Zl6d5779WwYcP0hz/8QevXr9evfvUrHTp0\nSEuWLJHL5dL+/fu1Zs0arVq1KhgWN2/erD179mj79u3auXOnDh06pA0bNkj67MH5ZWVlqqioUE1N\njdLS0lRaWipJamlpUUlJiaZPn66DBw9q5syZmjt3rnw+nyR1WhMAAABdE3YAPXbsmG644Qbt2bNH\n1dXVampq0n/913/p5ptv1oEDB7rUxFtvvaUPPvhACxcuVFJSkr7+9a9ry5Ytuuyyy7R7924tWLBA\nycnJGj16tPLy8rRt2zZJUlVVlWbNmqXBgwdr0KBBKi4u1vPPPy9J2rFjh3Jzc5WZmSmHw6GFCxdq\n37598nq9qqmpkd1uV1FRkZKSklRYWKi0tDTt3btXPp+v05oAAADomrAD6MqVKzVr1ixt2bIleMl9\nxYoVmjlzplatWtWlJo4ePaqvf/3reuSRR3Tdddfphhtu0FtvvaXGxkYlJiZq6NChwXMzMjJUV1cn\nSaqrq9Pw4cND1jwejyzL6rCWkpKi/v37y+PxyOPxyO12h/Rw7nVPnjzZaU0AAAB0Tdg3IR09elTL\nly/vcPyWW27Rpk2butREY2Oj/vCHP2jChAl65ZVX9M4772jOnDl6/PHH5XA4Qs51OBxqbm6WJPn9\n/pB1p9OpQCAQfDTU//1ap9Mpv9+vpqYmOZ3O875uU1NTpzXDYbPZZP9Cv9zwxdnttoufFIMSEuLr\nfZ2bU7zOKx4wo9jAnKIfM4oN0TKnsANo//79derUKaWnp4ccP3r0qFJTU7vURJ8+fdS/f38VFxdL\nksaMGaMbbrhBa9asUUtLS8i5zc3Ncrlckj4Lhp9f9/v9SkxMVHJy8nlDo9/vl8vlktPp7LB27nWd\nTmenNcMxcGDfDh9TivCkpvaLdAs9YsCAvpFuARfBjGIDc4p+zCg2RHpOYQfQ2267TT/72c+0cOFC\nSdKJEyf06quv6tFHH9Xs2bO71ERGRoba29vV3t6uhIQESVJ7e7uuvPJK1dbW6tSpUxoyZIgkyePx\nBC+tu91ueTweZWZmBteGDRsWsnZOQ0ODGhsb5Xa75fP5Ouzaejwe5eXlKT09XW1tbResGY4PP/Sx\nA3qJGhrORrqFbmW32zRgQF+dOeNTIGBFuh2cBzOKDcwp+jGj2GByTp1tKoUdQO+55x717dtXDz/8\nsPx+v+bPn6+0tDSVlJRo1qxZXWrwuuuuk8Ph0K9+9SvNmzdPR44c0e9//3s9+eST+p//+R+Vl5dr\n+fLleu+991RdXa3HH39ckjRt2jStX79eEyZMUGJiotatW6eCggJJUl5enu644w7NmDFDo0aNUkVF\nhSZNmqSUlBTl5OSotbVVGzdu1K233qqqqip5vV5NnDhRLpdLubm5F6wZDsuy9LmnQeELaG+Pzx9a\ngYAVt+8tXjCj2MCcoh8zig2RnpPNsqywqr/44ou69tpr9eUvf1lNTU1qb2/Xl770pW5r5OTJk/r5\nz3+ut99+W/369dO8efM0Y8YMnTlzRmVlZTpw4IBcLpfmz5+vwsJCSZ/tkq5Zs0bPPfec2tralJ+f\nr9LS0uAu6s6dO7V69WrV19dr7NixWrlypQYOHCjps8c0LV26VCdOnFB6erqWLl2qrKwsSeq0Zjjq\n6z/ptn8uF5KQYNOsFS/1eB3TNjw4NdItdKuEBJtSU/upoeEsP5CjFDOKDcwp+jGj2GByToMGXTgn\nhh1Ax48fr6effrrD3ePoiAB66QigMI0ZxQbmFP2YUWyIlgAa9m8qXnXVVXr11Ve7pSEAAAD0XmH/\nDmifPn30y1/+Ur/+9a/1ta99rcOjirZs2dLtzQEAACD+hB1Ar7rqKl111VU92QsAAAB6gU4D6Pjx\n4/Xiiy8qNTVV8+fPl/TZzTvDhg1Tnz59jDQIAACA+NLp74B+/PHH+r/3KBUVFelvf/tbjzYFAACA\n+PWFH5ce5k3zAAAAwHn18Of1AAAAAKEIoAAAADDqonfBV1VVqW/f//eB9YFAQNXV1UpNTQ0575Zb\nbun+7gAAABB3Og2gQ4YM0aZNm0KODRw4UFu3bg05ZrPZCKAAAAAIS6cB9OWXXzbVBwAAAHoJfgcU\nAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYR\nQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABg\nFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAA\nAEYRQAG/xGQSAAATX0lEQVQAAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEA\nAGAUARQAAABGEUABAABgFAEUAAAARhFAAQAAYFTUBVCv16ucnBy98sorkqTGxkbNmzdP2dnZmjJl\nirZu3Ro817IslZeXa8KECRo3bpyWL1+u9vb24Hp1dbVyc3OVlZWl4uJieb3e4NqxY8dUWFiorKws\nFRQU6PDhw8G1zmoCAACga6IugC5evFhnzpwJ/nnJkiVyuVzav3+/1qxZo1WrVgXD4ubNm7Vnzx5t\n375dO3fu1KFDh7RhwwZJ0vHjx1VWVqaKigrV1NQoLS1NpaWlkqSWlhaVlJRo+vTpOnjwoGbOnKm5\nc+fK5/NdtCYAAAC6JqoC6NNPPy2n06mvfOUrkiSfz6fdu3drwYIFSk5O1ujRo5WXl6dt27ZJkqqq\nqjRr1iwNHjxYgwYNUnFxsZ5//nlJ0o4dO5Sbm6vMzEw5HA4tXLhQ+/btk9frVU1Njex2u4qKipSU\nlKTCwkKlpaVp7969F60JAACArkmMdAPneDwePfnkk3r22Wc1ffp0SdLJkyeVmJiooUOHBs/LyMjQ\nrl27JEl1dXUaPnx4yJrH45FlWaqrq9PVV18dXEtJSVH//v3l8Xjk8XjkdrtD6mdkZKiurk5XXHFF\npzXDYbPZZO/haG+323q2QIQkJMTX+zo3p3idVzxgRrGBOUU/ZhQbomVOURFAP/30U/3kJz/R4sWL\nNWDAgODxpqYmORyOkHMdDoeam5slSX6/P2Td6XQqEAiotbW1w9q5db/fr6amJjmdzvO+7sVqhmPg\nwL6y2fgGvBSpqf0i3UKPGDCgb6RbwEUwo9jAnKIfM4oNkZ5TVATQf//3f9fIkSM1efLkkONOp1Mt\nLS0hx5qbm+VyuSR9Fgw/v+73+5WYmKjk5OTzhka/3y+XyyWn09lh7dzrXqxmOD780McO6CVqaDgb\n6Ra6ld1u04ABfXXmjE+BgBXpdnAezCg2MKfox4xig8k5dbapFBUBdOfOnaqvr9fOnTslSWfPntWP\nf/xjzZkzR21tbTp16pSGDBki6bNL9ecuu7vdbnk8HmVmZgbXhg0bFrJ2TkNDgxobG+V2u+Xz+bRp\n06aQHjwej/Ly8pSent5pzXBYlqXP3YyPL6C9PT5/aAUCVty+t3jBjGIDc4p+zCg2RHpOUXET0osv\nvqg33nhDtbW1qq2t1ZAhQ1RRUaF58+YpNzdX5eXl8vv9OnLkiKqrq5Wfny9JmjZtmtavX6/Tp0/L\n6/Vq3bp1KigokCTl5eVp165dqq2tVUtLiyoqKjRp0iSlpKQoJydHra2t2rhxo9ra2lRZWSmv16uJ\nEyeqX79+ndYEAABA10RFAO3MsmXL9Omnn2ry5MlasGCBFi1aFNzxLCoq0tSpU1VYWKjvf//7GjNm\njGbPni1JGjlypJYtW6bFixcrJydHH3zwgVauXClJ6tOnj5544gn97ne/0/jx47Vp0yatXbs2eJm9\ns5oAAADoGptlWeyTd7P6+k96vEZCgk2zVrzU43VM2/Dg1Ei30K0SEmxKTe2nhoazXJKKUswoNjCn\n6MeMYoPJOQ0a9KULrkX9DigAAADiCwEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUAB\nAABgFAEUAAAARhFAAQAAYBQBFAAAAEYRQAEAAGAUARQAAABGEUABAABgFAEUAAAARiVGugHg8+56\n+OVIt9DtdpQXRLoFAACiCjugAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowig\nAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCK\nAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAA\nowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAA\nADAqagJobW2tbr75ZmVnZ+sf/uEftGXLFklSY2Oj5s2bp+zsbE2ZMkVbt24Nfo1lWSovL9eECRM0\nbtw4LV++XO3t7cH16upq5ebmKisrS8XFxfJ6vcG1Y8eOqbCwUFlZWSooKNDhw4eDa53VBAAAQNdE\nRQBtbGzUvffeqzvvvFMHDx7U6tWrVVFRof3792vJkiVyuVzav3+/1qxZo1WrVgXD4ubNm7Vnzx5t\n375dO3fu1KFDh7RhwwZJ0vHjx1VWVqaKigrV1NQoLS1NpaWlkqSWlhaVlJRo+vTpOnjwoGbOnKm5\nc+fK5/NJUqc1AQAA0DWJkW5Akk6dOqXJkycrPz9fkvTNb35T11xzjQ4dOqTdu3frP//zP5WcnKzR\no0crLy9P27ZtU1ZWlqqqqjRr1iwNHjxYklRcXKzVq1fr7rvv1o4dO5Sbm6vMzExJ0sKFC5WTkyOv\n16ujR4/KbrerqKhIklRYWKjf/OY32rt3ryZPntxpTeCLyr+/KtItdLsND06NdAsAgBgWFQF05MiR\neuSRR4J/bmxsVG1trf7+7/9eiYmJGjp0aHAtIyNDu3btkiTV1dVp+PDhIWsej0eWZamurk5XX311\ncC0lJUX9+/eXx+ORx+OR2+0O6SEjI0N1dXW64oorOq0ZDpvNJnsP7y3b7baeLQB0IiEhfv79O/e9\nxPdUdGNO0Y8ZxYZomVNUBNDP++STT1RSUhLcBX3qqadC1h0Oh5qbmyVJfr9fDocjuOZ0OhUIBNTa\n2tph7dy63+9XU1OTnE7neV+3qampw9d9vmY4Bg7sK5uNb0DEr9TUfpFuodsNGNA30i0gDMwp+jGj\n2BDpOUVVAP3rX/+qkpISDR06VP/2b/+mP//5z2ppaQk5p7m5WS6XS9JnwfDz636/X4mJiUpOTj5v\naPT7/XK5XHI6nR3Wzr2u0+nstGY4PvzQxw4o4lpDw9lIt9Bt7HabBgzoqzNnfAoErEi3gwtgTtGP\nGcUGk3PqbLMiagLo0aNHNWfOHE2bNk0PPPCA7Ha70tPT1dbWplOnTmnIkCGSJI/HE7zs7na75fF4\ngr/n6fF4NGzYsJC1cxoaGtTY2Ci32y2fz6dNmzaF1Pd4PMrLy7tozXBYlqXP3YwPxJ329vj7yyUQ\nsOLyfcUb5hT9mFFsiPScouIueK/Xqzlz5mj27NkqLS2V/f/fPuzXr59yc3NVXl4uv9+vI0eOqLq6\nOniz0rRp07R+/XqdPn1aXq9X69atU0FBgSQpLy9Pu3btUm1trVpaWlRRUaFJkyYpJSVFOTk5am1t\n1caNG9XW1qbKykp5vV5NnDjxojUBAADQNVGxA1pZWamGhgatXbtWa9euDR6/8847tWzZMpWVlWny\n5MlyuVxatGhRcMezqKhIXq9XhYWFamtrU35+vmbPni3psxubli1bpsWLF6u+vl5jx47VypUrJUl9\n+vTRE088oaVLl6qiokLp6elau3Zt8DJ7ZzUBAADQNTbLstgn72b19Z/0eI2EBJtmrXipx+sA5xNP\nj2FKSLApNbWfGhrOctkwijGn6MeMYoPJOQ0a9KULrkXFJXgAAAD0HgRQAAAAGEUABQAAgFEEUAAA\nABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQAAAAGEUABQAAgFFR8VGcAGLLXQ+/HOkWutWO8oJItwAA\nvQo7oAAAADCKAAoAAACjCKAAAAAwigAKAAAAowigAAAAMIoACgAAAKMIoAAAADCKAAoAAACjCKAA\nAAAwigAKAAAAo/goTgC9Xv79VZFuodtteHBqpFsAgAtiBxQAAABGEUABAABgFAEUAAAARhFAAQAA\nYBQBFAAAAEYRQAEAAGAUj2ECgDh018MvR7qFbrejvCDSLQDoJuyAAgAAwCgCKAAAAIwigAIAAMAo\nAigAAACMIoACAADAKO6CBwDEhPz7qyLdQrfb8ODUSLcARAQ7oAAAADCKAAoAAACjCKAAAAAwigAK\nAAAAowigAAAAMIq74AEAAC7grodfjnQL3W5HeUGkWyCAAgAQKfEWbqIh2CA2cAkeAAAARhFAAQAA\nYBQBFAAAAEYRQAEAAGAUNyEBAIBukX9/VaRbQIxgBxQAAABGEUABAABgFAH0Ao4dO6bCwkJlZWWp\noKBAhw8fjnRLAAAAcYEAeh4tLS0qKSnR9OnTdfDgQc2cOVNz586Vz+eLdGsAAAAxjwB6HjU1NbLb\n7SoqKlJSUpIKCwuVlpamvXv3Rro1AACAmEcAPQ+PxyO32x1yLCMjQ3V1dRHqCAAAIH7wGKbzaGpq\nktPpDDnmcDjU3Nwc1tfbbDbZezja2+22ni0AAADiVqRzBAH0PJxOZ4ew2dzcLJfLFdbXp6X164m2\nOthRXmCkDgAAiC8DBvSNaH0uwZ/HsGHD5PF4Qo55PB4NHz48Qh0BAADEDwLoeeTk5Ki1tVUbN25U\nW1ubKisr5fV6NXHixEi3BgAAEPNslmVZkW4iGh0/flxLly7ViRMnlJ6erqVLlyorKyvSbQEAAMQ8\nAigAAACM4hI8AAAAjCKAAgAAwCgCKAAAAIwigAIAAMAoAmiUO3bsmAoLC5WVlaWCggIdPnz4vOdV\nV1crNzdXWVlZKi4ultfrNdxp7xXujJ599ll95zvf0ZgxYzRjxgzV1tYa7rR3C3dO5xw4cEAjRoyQ\nz+cz1CHCnVFtba1uuukmXX311crPz9eBAwcMd9q7hTunrVu3Kjc3V9nZ2br11lv1zjvvGO4UR44c\n6fQRkhHNDhaiVnNzs/Wtb33L2rx5s9Xa2mpt3brVmjBhgnX27NmQ8959911rzJgx1uHDhy2/32/9\n9Kc/tebMmROhrnuXcGd04MAB65prrrGOHTtmtbe3W//xH/9hZWdnWw0NDRHqvHcJd07nnDlzxpoy\nZYr1jW9844LnoHuFO6PTp09bY8eOtV588UUrEAhYO3bssLKzsy2/3x+hznuXL/L30vjx4626ujqr\nvb3dWrdunTV16tQIdd37BAIBa+vWrVZ2drY1fvz4854T6ezADmgUq6mpkd1uV1FRkZKSklRYWKi0\ntDTt3bs35LwdO3YoNzdXmZmZcjgcWrhwofbt28cuqAHhzuj06dP6x3/8R40cOVJ2u1033XSTEhIS\n9Kc//SlCnfcu4c7pnKVLl+p73/ue4S57t3BnVFVVpWuvvVY33HCDbDab8vLy9Jvf/EZ2O3+dmRDu\nnE6ePKlAIKD29nZZliW73S6HwxGhrnufxx57TE899ZRKSkoueE6kswPfsVHM4/HI7XaHHMvIyFBd\nXV3Isbq6upCPCU1JSVH//v07fJwoul+4M7rxxht19913B//8xhtvyOfzdfha9Ixw5yRJ27dv18cf\nf6zbbrvNVHtQ+DM6evSoLrvsMs2bN0/XXHONbrnlFrW3t6tPnz4m2+21wp3TxIkTdcUVV+j73/++\nRo0apXXr1mnVqlUmW+3VZsyYoaqqKo0aNeqC50Q6OxBAo1hTU5OcTmfIMYfDoebm5pBjfr+/w39Z\nOp1O+f3+Hu+xtwt3Rp/3pz/9SQsWLNCCBQuUmpra0y1C4c/p1KlTWr16tX7xi1+YbA8Kf0aNjY3a\nunWrbrvtNr322muaNm2a7rnnHjU2Nppst9cKd04tLS0aPny4Kisr9eabb2rWrFmaP39+pz8b0X0G\nDx4sm83W6TmRzg4E0CjmdDo7fLM2NzfL5XKFHLtQKP2/56H7hTujc1577TXddtttuv3223XPPfeY\naBEKb06BQEAPPPCAfvSjH+myyy4z3WKvF+73Up8+fTRp0iRNnDhRSUlJuv322+VyuXTo0CGT7fZa\n4c7pV7/6lS6//HKNGjVKycnJmjdvntra2rR//36T7aITkc4OBNAoNmzYsA5b4R6PJ2TLXJLcbnfI\neQ0NDWpsbOTyrgHhzkiSnnvuOS1YsEBlZWW69957TbUIhTen06dP66233tLSpUs1duxYTZs2TZI0\nefJknlhgQLjfSxkZGWptbQ05FggEZPGp0kaEO6dTp06FzMlmsykhIUEJCQlG+sTFRTo7EECjWE5O\njlpbW7Vx40a1tbWpsrJSXq+3wyMV8vLytGvXLtXW1qqlpUUVFRWaNGmSUlJSItR57xHujA4cOKCH\nHnpIjz/+uPLy8iLUbe8VzpyGDBmiI0eOqLa2VrW1tdq+fbskae/evRo7dmykWu81wv1eKigo0Guv\nvaY9e/YoEAho48aNamlp0TXXXBOhznuXcOc0ZcoUVVZW6ujRo/r000/15JNPqr29XdnZ2RHqHP9X\nxLODsfvtcUneffdd65ZbbrGysrKsgoIC680337Qsy7KWLFliLVmyJHje7373O+s73/mOdfXVV1t3\n33235fV6I9VyrxPOjGbPnm2NGDHCysrKCvnf3r17I9l6rxLu99I5f/3rX3kMk2Hhzmjfvn1WQUGB\nlZWVZd10003W4cOHI9VyrxTOnAKBgLVu3Trr+uuvt7Kzs6077rjDOnHiRCTb7pVqampCHsMUTdnB\nZllctwAAAIA5XIIHAACAUQRQAAAAGEUABQAAgFEEUAAAABhFAAUAAIBRBFAAAAAYRQAFAACAUQRQ\nAAAAGEUABQAAgFH/H9L2B+yPAh46AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe182c92f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(features_test).plot.hist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
